{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python395jvsc74a57bd0e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a",
   "display_name": "Python 3.9.5 64-bit"
  },
  "metadata": {
   "interpreter": {
    "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# Metrics\n",
    "\n",
    "Evaluates a trained model accordingly to the metrics specified on the paper"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../src/\")\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, multilabel_confusion_matrix \n",
    "from sklearn.model_selection import train_test_split\n",
    "from ModelHandler import ModelHandler\n",
    "import pickle\n",
    "import h5py\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score     \n",
    "from tqdm import tqdm\n",
    "\n",
    "configs = {\n",
    "    \"N_GRIDS\": 5, \n",
    "    \"SIGNAL_BASE_LENGTH\": 12800, \n",
    "    \"N_CLASS\": 26, \n",
    "    \"USE_NO_LOAD\": False, \n",
    "    \"AUGMENTATION_RATIO\": 5, \n",
    "    \"MARGIN_RATIO\": 0.15, \n",
    "    \"DATASET_PATH\": \"../Synthetic_Full_iHall.hdf5\",\n",
    "    \"TRAIN_SIZE\": 0.8,\n",
    "    \"FOLDER_PATH\": \"../TrainedWeights/Final/\", \n",
    "    \"FOLDER_DATA_PATH\": \"../TrainedWeights/Final/\", \n",
    "    \"N_EPOCHS_TRAINING\": 250,\n",
    "    \"INITIAL_EPOCH\": 0,\n",
    "    \"TOTAL_MAX_EPOCHS\": 250,\n",
    "    \"SNRdb\": None # Noise level on db\n",
    "}\n",
    "\n",
    "\n",
    "folderPath = configs[\"FOLDER_PATH\"]\n",
    "folderDataPath = configs[\"FOLDER_DATA_PATH\"]\n",
    "signalBaseLength = configs[\"SIGNAL_BASE_LENGTH\"]\n",
    "ngrids = configs[\"N_GRIDS\"]\n",
    "trainSize = configs[\"TRAIN_SIZE\"]\n",
    "\n",
    "dict_data = pickle.load(open(folderDataPath + \"sorted_aug_data_\" + str(ngrids) + \"_\" + str(signalBaseLength) + \".p\", \"rb\")) # Load data\n",
    "x_train = dict_data[\"x_train\"]\n",
    "x_test = dict_data[\"x_test\"]\n",
    "y_train = dict_data[\"y_train\"]\n",
    "y_test = dict_data[\"y_test\"]"
   ]
  },
  {
   "source": [
    "## Evaluates the identification\n",
    "\n",
    "This step generates a dict with the ground truth and the prediction for each test example"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "100%|██████████| 10/10 [14:23<00:00, 86.34s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "X_all = np.vstack((x_train, x_test))\n",
    "ydet_all = np.vstack((y_train[\"detection\"], y_test[\"detection\"]))\n",
    "ytype_all = np.vstack((y_train[\"type\"], y_test[\"type\"]))\n",
    "yclass_all = np.vstack((y_train[\"classification\"], y_test[\"classification\"]))\n",
    "\n",
    "final_acc_on, final_acc_off, final_acc = [], [], []\n",
    "y = {}\n",
    "for fold in tqdm(range(1, 11)):\n",
    "    foldFolderPath = folderPath + str(fold) + \"/\"\n",
    "    \n",
    "    train_index = np.load(foldFolderPath + \"train_index.npy\")\n",
    "    test_index = np.load(foldFolderPath + \"test_index.npy\")\n",
    "\n",
    "    bestModel = ModelHandler.loadModel(foldFolderPath + \"best_model.h5\", type_weights=None) # Load model\n",
    "\n",
    "    x_train = X_all[train_index]\n",
    "    x_test = X_all[test_index]\n",
    "    y_train[\"detection\"] = ydet_all[train_index]\n",
    "    y_test[\"detection\"] = ydet_all[test_index]\n",
    "    y_train[\"type\"] = ytype_all[train_index]\n",
    "    y_test[\"type\"] = ytype_all[test_index]\n",
    "    y_train[\"classification\"] = yclass_all[train_index]\n",
    "    y_test[\"classification\"] = yclass_all[test_index]\n",
    "\n",
    "    final_prediction = []\n",
    "    final_groundTruth = []\n",
    "    for xi, yclass, ytype in zip(x_test, y_test[\"classification\"], y_test[\"type\"]):\n",
    "        pred = bestModel.predict(np.expand_dims(xi, axis=0))\n",
    "        prediction = np.max(pred[2][0],axis=0)\n",
    "        groundTruth = np.max(yclass,axis=0)\n",
    "\n",
    "        final_prediction.append(prediction)\n",
    "        final_groundTruth.append(groundTruth) \n",
    "\n",
    "        del xi, yclass, ytype\n",
    "\n",
    "    y[fold] = {}\n",
    "    y[fold][\"true\"] = final_groundTruth.copy()\n",
    "    y[fold][\"pred\"] = final_prediction.copy()"
   ]
  },
  {
   "source": [
    "### F1 Score\n",
    "\n",
    "#### F1 Macro:\n",
    "$$\n",
    "\\begin{gather*}\n",
    "F1_{Macro} = \\frac{1}{Y} \\sum_{i=1}^{Y} \\frac{2 \\cdot tp_i}{2 \\cdot tp_i + fp_i + fn_i}\n",
    "\\end{gather*}\n",
    "$$\n",
    "\n",
    "#### F1 Micro:\n",
    "$$\n",
    "\\begin{gather*}\n",
    "F1_{Micro} = \\frac{2 \\cdot \\sum_{i=1}^{Y} tp_i}{\\sum_{i=1}^{Y} 2 \\cdot tp_i + fp_i + fn_i}\n",
    "\\end{gather*}\n",
    "$$\n",
    "\n",
    "- $tp_i$: True positives classifications for appliance $i$\n",
    "- $fp_i$: False positives classifications for appliance $i$\n",
    "- $fn_i$: False negatives classifications for appliance $i$"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Fold 1: F1 Macro: 96.9, F1 Micro: 97.3\nFold 2: F1 Macro: 96.6, F1 Micro: 96.3\nFold 3: F1 Macro: 96.5, F1 Micro: 96.5\nFold 4: F1 Macro: 97.1, F1 Micro: 97.1\nFold 5: F1 Macro: 97.8, F1 Micro: 97.4\nFold 6: F1 Macro: 96.8, F1 Micro: 97.2\nFold 7: F1 Macro: 96.8, F1 Micro: 96.6\nFold 8: F1 Macro: 96.5, F1 Micro: 96.5\nFold 9: F1 Macro: 97.6, F1 Micro: 97.1\nFold 10: F1 Macro: 95.5, F1 Micro: 96.1\nAverage: F1 Macro: 96.8, F1 Micro: 96.8\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score   \n",
    "from PostProcessing import PostProcessing\n",
    "\n",
    "postProcessing = PostProcessing(configs=configs)\n",
    "\n",
    "threshold = 0.5\n",
    "f1_macro, f1_micro = [], []\n",
    "for fold in range(1, 11):\n",
    "    f1_macro.append(f1_score(np.array(y[fold][\"true\"]) > threshold, np.array(y[fold][\"pred\"]) > threshold, average='macro'))\n",
    "    f1_micro.append(f1_score(np.array(y[fold][\"true\"]) > threshold, np.array(y[fold][\"pred\"]) > threshold, average='micro'))\n",
    "    print(f\"Fold {fold}: F1 Macro: {f1_macro[-1] * 100:.1f}, F1 Micro: {f1_micro[-1] * 100:.1f}\")\n",
    "\n",
    "print(f\"Average: F1 Macro: {np.average(f1_macro) * 100:.1f}, F1 Micro: {np.average(f1_micro) * 100:.1f}\")"
   ]
  },
  {
   "source": [
    "### Accuracy (ACC)\n",
    "\n",
    "$$\n",
    "\\begin{gather*}\n",
    "ACC_i = \\frac{CCE_i}{TNE_i} \\\\ \\\\\n",
    "ACC = \\frac{1}{Y} \\sum_{i = 1}^{Y} ACC_i\n",
    "\\end{gather*}\n",
    "$$\n",
    "\n",
    "- $ACC_i$: Accuracy for appliance $i$\n",
    "- $CCE_i$: Load connected successfully identified\n",
    "- $TNE_i$: Total of connected events"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Fold 1, Acc on: 99.2, Acc off: 97.6, Acc no event: 95.4 Acc total: 96.7\n",
      "Fold 2, Acc on: 95.7, Acc off: 93.8, Acc no event: 97.4 Acc total: 97.0\n",
      "Fold 3, Acc on: 95.6, Acc off: 97.6, Acc no event: 96.9 Acc total: 96.9\n",
      "Fold 4, Acc on: 97.3, Acc off: 96.8, Acc no event: 97.1 Acc total: 97.1\n",
      "Fold 5, Acc on: 99.0, Acc off: 97.9, Acc no event: 98.5 Acc total: 98.5\n",
      "<ipython-input-15-ff2f1a66a2ac>:33: RuntimeWarning: invalid value encountered in true_divide\n",
      "  acc_off.append(100 * np.average(np.nan_to_num(correct_off/total_off)))\n",
      "Fold 6, Acc on: 97.3, Acc off: 97.5, Acc no event: 96.7 Acc total: 97.0\n",
      "Fold 7, Acc on: 97.8, Acc off: 97.6, Acc no event: 96.8 Acc total: 97.1\n",
      "Fold 8, Acc on: 95.8, Acc off: 94.7, Acc no event: 97.7 Acc total: 96.7\n",
      "Fold 9, Acc on: 96.9, Acc off: 94.9, Acc no event: 97.6 Acc total: 97.7\n",
      "Fold 10, Acc on: 95.6, Acc off: 92.4, Acc no event: 96.3 Acc total: 95.8\n",
      "Total, Acc on: 97.0, Acc off: 96.1, Acc no event: 97.0, Acc total: 97.0\n"
     ]
    }
   ],
   "source": [
    "threshold = 0.5\n",
    "\n",
    "ytype_all = np.vstack((y_train[\"type\"], y_test[\"type\"]))\n",
    "\n",
    "acc_on, acc_off, acc_no_event, acc_total = [], [], [], []\n",
    "for fold in range(1, 11):\n",
    "    correct_on = np.zeros((26,1))\n",
    "    total_on = np.zeros((26,1))\n",
    "    correct_off = np.zeros((26,1))\n",
    "    total_off = np.zeros((26,1))\n",
    "    correct_no_event = np.zeros((26,1))\n",
    "    total_no_event = np.zeros((26,1))\n",
    "\n",
    "    train_index = np.load(folderPath + str(fold) + \"/train_index.npy\")\n",
    "    test_index = np.load(folderPath + str(fold) + \"/test_index.npy\")\n",
    "\n",
    "    ytype_train = ytype_all[train_index]\n",
    "    ytype_test = ytype_all[test_index]\n",
    "\n",
    "    for ytype, ytrue, ypred in zip(ytype_test, y[fold][\"true\"], y[fold][\"pred\"]):\n",
    "        event_type = np.min(np.argmax(ytype, axis=1))\n",
    "        if event_type == 0:\n",
    "            correct_on[np.bitwise_and(ytrue > threshold, ypred > threshold)] += 1\n",
    "            total_on[ytrue > threshold] += 1\n",
    "        elif event_type == 1:\n",
    "            correct_off[np.bitwise_and(ytrue > threshold, ypred > threshold)] += 1\n",
    "            total_off[ytrue > threshold] += 1\n",
    "        else:\n",
    "            correct_no_event[np.bitwise_and(ytrue > threshold, ypred > threshold)] += 1\n",
    "            total_no_event[ytrue > threshold] += 1\n",
    "    \n",
    "    acc_on.append(100 * np.average(np.nan_to_num(correct_on/total_on)))\n",
    "    acc_off.append(100 * np.average(np.nan_to_num(correct_off/total_off)))\n",
    "    acc_no_event.append(100 * np.average(np.nan_to_num(correct_no_event/total_no_event)))\n",
    "    acc_total.append(100 * np.average(np.nan_to_num((correct_on + correct_off + correct_no_event)/(total_on + total_off + total_no_event))))\n",
    "\n",
    "    print(f\"Fold {fold}, Acc on: {acc_on[-1]:.1f}, Acc off: {acc_off[-1]:.1f}, Acc no event: {acc_no_event[-1]:.1f} Acc total: {acc_total[-1]:.1f}\")\n",
    "\n",
    "print(f\"Total, Acc on: {np.average(acc_on):.1f}, Acc off: {np.average(acc_off):.1f}, Acc no event: {np.average(acc_no_event):.1f}, Acc total: {np.average(acc_total):.1f}\")"
   ]
  },
  {
   "source": [
    "## Detection Metrics\n",
    "\n",
    "### D\n",
    "$$\n",
    "\\begin{gather*}\n",
    "D = \\frac{ \\sum_{i=1}^{A} |d(i) - ev(i)|}{A}\n",
    "\\end{gather*}\n",
    "$$\n",
    "\n",
    "- `A`: Total of events correctly detected ($\\pm$ 10 semi cycles tolerance)\n",
    "- `d(i)`: Detection for appliance $i$\n",
    "- `ev(i)`: Ground truth detection for appliance $i$\n",
    "\n",
    "## PC\n",
    "\n",
    "$$\n",
    "\\begin{gather*}\n",
    "PC = \\frac{A}{N}\n",
    "\\end{gather*}\n",
    "$$\n",
    "\n",
    "- `A`: Total of events correctly detected ($\\pm$ 10 semi cycles tolerance)\n",
    "- `N`: Total of events"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "-------------- FOLD 1 ---------------\n",
      "Total time: 108.65421447705921, Average Time: 0.05208735113952982\n",
      "LIT-SYN-1 PCmetric: (1.0, 1.0, 1.0)\n",
      "LIT-SYN-1 Dmetric: (0.7837837837837838, 0.71875, 0.7536231884057971)\n",
      "LIT-SYN-2 PCmetric: (0.991304347826087, 0.9791666666666666, 0.9845559845559846)\n",
      "LIT-SYN-2 Dmetric: (0.7456140350877193, 0.6808510638297872, 0.7098039215686275)\n",
      "LIT-SYN-3 PCmetric: (0.9870967741935484, 0.9840425531914894, 0.9854227405247813)\n",
      "LIT-SYN-3 Dmetric: (0.8431372549019608, 0.8324324324324325, 0.8372781065088757)\n",
      "LIT-SYN-8 PCmetric: (0.9142857142857143, 0.9324324324324325, 0.9236111111111112)\n",
      "LIT-SYN-8 Dmetric: (1.25, 1.1014492753623188, 1.1729323308270676)\n",
      "LIT-SYN-All PCmetric: (0.9761273209549072, 0.9748858447488584, 0.9754601226993865)\n",
      "LIT-SYN-All Dmetric: (0.8777173913043478, 0.8173302107728337, 0.8452830188679246)\n",
      "-------------- FOLD 2 ---------------\n",
      "Total time: 85.39059834195359, Average Time: 0.04081768563190898\n",
      "LIT-SYN-1 PCmetric: (0.96875, 1.0, 0.9855072463768116)\n",
      "LIT-SYN-1 Dmetric: (1.032258064516129, 0.5675675675675675, 0.7794117647058824)\n",
      "LIT-SYN-2 PCmetric: (0.9767441860465116, 0.993006993006993, 0.9852941176470589)\n",
      "LIT-SYN-2 Dmetric: (0.7380952380952381, 0.5915492957746479, 0.6604477611940298)\n",
      "LIT-SYN-3 PCmetric: (0.9430379746835443, 0.9820359281437125, 0.963076923076923)\n",
      "LIT-SYN-3 Dmetric: (0.9328859060402684, 0.7926829268292683, 0.8594249201277955)\n",
      "LIT-SYN-8 PCmetric: (0.9, 0.9444444444444444, 0.9222222222222223)\n",
      "LIT-SYN-8 Dmetric: (1.1604938271604939, 0.9294117647058824, 1.0421686746987953)\n",
      "LIT-SYN-All PCmetric: (0.9462102689486552, 0.9794050343249427, 0.9633569739952719)\n",
      "LIT-SYN-All Dmetric: (0.9250645994832042, 0.7336448598130841, 0.8245398773006135)\n",
      "-------------- FOLD 3 ---------------\n",
      "Total time: 68.43702808897251, Average Time: 0.033221858295617725\n",
      "LIT-SYN-1 PCmetric: (1.0, 1.0, 1.0)\n",
      "LIT-SYN-1 Dmetric: (0.6415094339622641, 0.7142857142857143, 0.6704545454545454)\n",
      "LIT-SYN-2 PCmetric: (0.9862068965517241, 0.9838709677419355, 0.9851301115241635)\n",
      "LIT-SYN-2 Dmetric: (0.7762237762237763, 0.6475409836065574, 0.7169811320754716)\n",
      "LIT-SYN-3 PCmetric: (0.9533333333333334, 0.9938271604938271, 0.9743589743589743)\n",
      "LIT-SYN-3 Dmetric: (0.8951048951048951, 0.7950310559006211, 0.8421052631578947)\n",
      "LIT-SYN-8 PCmetric: (0.8921568627450981, 0.8192771084337349, 0.8594594594594595)\n",
      "LIT-SYN-8 Dmetric: (1.164835164835165, 0.8676470588235294, 1.0377358490566038)\n",
      "LIT-SYN-All PCmetric: (0.9555555555555556, 0.9554455445544554, 0.955503512880562)\n",
      "LIT-SYN-All Dmetric: (0.8813953488372093, 0.7538860103626943, 0.821078431372549)\n",
      "-------------- FOLD 4 ---------------\n",
      "Total time: 69.42583364703569, Average Time: 0.03353905007103173\n",
      "LIT-SYN-1 PCmetric: (1.0, 1.0, 1.0)\n",
      "LIT-SYN-1 Dmetric: (0.6111111111111112, 0.6486486486486487, 0.6301369863013698)\n",
      "LIT-SYN-2 PCmetric: (0.9694656488549618, 0.9852941176470589, 0.9775280898876404)\n",
      "LIT-SYN-2 Dmetric: (0.7559055118110236, 0.7835820895522388, 0.7701149425287356)\n",
      "LIT-SYN-3 PCmetric: (0.9698795180722891, 0.9626865671641791, 0.9666666666666667)\n",
      "LIT-SYN-3 Dmetric: (0.8509316770186336, 0.8062015503875969, 0.8310344827586207)\n",
      "LIT-SYN-8 PCmetric: (0.8783783783783784, 0.9130434782608695, 0.8975903614457831)\n",
      "LIT-SYN-8 Dmetric: (1.0, 1.0, 1.0)\n",
      "LIT-SYN-All PCmetric: (0.9557739557739557, 0.9624060150375939, 0.9590570719602978)\n",
      "LIT-SYN-All Dmetric: (0.8226221079691517, 0.8255208333333334, 0.8240620957309185)\n",
      "-------------- FOLD 5 ---------------\n",
      "Total time: 70.89029163703526, Average Time: 0.03347039265204686\n",
      "LIT-SYN-1 PCmetric: (1.0, 1.0, 1.0)\n",
      "LIT-SYN-1 Dmetric: (0.72, 0.5454545454545454, 0.6206896551724138)\n",
      "LIT-SYN-2 PCmetric: (0.9710982658959537, 0.9864864864864865, 0.9781931464174455)\n",
      "LIT-SYN-2 Dmetric: (0.7202380952380952, 0.6027397260273972, 0.6656050955414012)\n",
      "LIT-SYN-3 PCmetric: (0.9305555555555556, 0.9873417721518988, 0.9602649006622517)\n",
      "LIT-SYN-3 Dmetric: (0.7910447761194029, 0.7692307692307693, 0.7793103448275862)\n",
      "LIT-SYN-8 PCmetric: (0.898876404494382, 0.9347826086956522, 0.9171270718232044)\n",
      "LIT-SYN-8 Dmetric: (0.9125, 1.1162790697674418, 1.0180722891566265)\n",
      "LIT-SYN-All PCmetric: (0.9443155452436195, 0.9767981438515081, 0.9605568445475638)\n",
      "LIT-SYN-All Dmetric: (0.7813267813267813, 0.7648456057007126, 0.7729468599033816)\n",
      "-------------- FOLD 6 ---------------\n",
      "Total time: 69.40542744799131, Average Time: 0.033384043986527806\n",
      "LIT-SYN-1 PCmetric: (1.0, 0.984375, 0.9911504424778761)\n",
      "LIT-SYN-1 Dmetric: (0.7346938775510204, 0.6190476190476191, 0.6696428571428571)\n",
      "LIT-SYN-2 PCmetric: (0.9739130434782609, 1.0, 0.9875)\n",
      "LIT-SYN-2 Dmetric: (0.75, 0.584, 0.6624472573839663)\n",
      "LIT-SYN-3 PCmetric: (0.8888888888888888, 1.0, 0.9413680781758957)\n",
      "LIT-SYN-3 Dmetric: (0.8402777777777778, 0.7517241379310344, 0.7958477508650519)\n",
      "LIT-SYN-8 PCmetric: (0.9146341463414634, 0.8888888888888888, 0.901840490797546)\n",
      "LIT-SYN-8 Dmetric: (0.96, 1.0694444444444444, 1.0136054421768708)\n",
      "LIT-SYN-All PCmetric: (0.9313725490196079, 0.9759036144578314, 0.9538274605103281)\n",
      "LIT-SYN-All Dmetric: (0.8236842105263158, 0.7358024691358025, 0.778343949044586)\n",
      "-------------- FOLD 7 ---------------\n",
      "Total time: 69.76811208600702, Average Time: 0.03328631301813312\n",
      "LIT-SYN-1 PCmetric: (0.975, 1.0, 0.9888888888888889)\n",
      "LIT-SYN-1 Dmetric: (0.9230769230769231, 0.62, 0.7528089887640449)\n",
      "LIT-SYN-2 PCmetric: (0.9854014598540146, 0.9703703703703703, 0.9779411764705882)\n",
      "LIT-SYN-2 Dmetric: (0.8740740740740741, 0.6106870229007634, 0.7443609022556391)\n",
      "LIT-SYN-3 PCmetric: (0.9390243902439024, 0.9886363636363636, 0.9647058823529412)\n",
      "LIT-SYN-3 Dmetric: (0.8831168831168831, 0.8103448275862069, 0.8445121951219512)\n",
      "LIT-SYN-8 PCmetric: (0.8723404255319149, 0.9069767441860465, 0.8888888888888888)\n",
      "LIT-SYN-8 Dmetric: (1.0121951219512195, 1.0512820512820513, 1.03125)\n",
      "LIT-SYN-All PCmetric: (0.9425287356321839, 0.9686800894854586, 0.95578231292517)\n",
      "LIT-SYN-All Dmetric: (0.9097560975609756, 0.7713625866050808, 0.8386714116251482)\n",
      "-------------- FOLD 8 ---------------\n",
      "Total time: 68.7343885879909, Average Time: 0.033029499561744786\n",
      "LIT-SYN-1 PCmetric: (0.9811320754716981, 1.0, 0.990909090909091)\n",
      "LIT-SYN-1 Dmetric: (0.6346153846153846, 0.5964912280701754, 0.6146788990825688)\n",
      "LIT-SYN-2 PCmetric: (1.0, 0.9765625, 0.9884169884169884)\n",
      "LIT-SYN-2 Dmetric: (0.6641221374045801, 0.712, 0.6875)\n",
      "LIT-SYN-3 PCmetric: (0.9345238095238095, 0.9727891156462585, 0.9523809523809523)\n",
      "LIT-SYN-3 Dmetric: (0.7133757961783439, 0.7062937062937062, 0.71)\n",
      "LIT-SYN-8 PCmetric: (0.8913043478260869, 0.9204545454545454, 0.9055555555555556)\n",
      "LIT-SYN-8 Dmetric: (0.926829268292683, 0.8024691358024691, 0.8650306748466258)\n",
      "LIT-SYN-All PCmetric: (0.9504504504504504, 0.9666666666666667, 0.9583333333333334)\n",
      "LIT-SYN-All Dmetric: (0.7298578199052133, 0.7118226600985221, 0.7210144927536232)\n",
      "-------------- FOLD 9 ---------------\n",
      "Total time: 68.74934173395741, Average Time: 0.033373466861144375\n",
      "LIT-SYN-1 PCmetric: (0.975, 1.0, 0.984375)\n",
      "LIT-SYN-1 Dmetric: (0.6666666666666666, 0.5416666666666666, 0.6190476190476191)\n",
      "LIT-SYN-2 PCmetric: (0.9769230769230769, 0.9925373134328358, 0.9848484848484849)\n",
      "LIT-SYN-2 Dmetric: (0.6299212598425197, 0.631578947368421, 0.6307692307692307)\n",
      "LIT-SYN-3 PCmetric: (0.9620253164556962, 0.9795918367346939, 0.9704918032786886)\n",
      "LIT-SYN-3 Dmetric: (0.7828947368421053, 0.7708333333333334, 0.777027027027027)\n",
      "LIT-SYN-8 PCmetric: (0.9195402298850575, 0.8543689320388349, 0.8842105263157894)\n",
      "LIT-SYN-8 Dmetric: (0.875, 0.7613636363636364, 0.8154761904761905)\n",
      "LIT-SYN-All PCmetric: (0.9590361445783132, 0.9534313725490197, 0.9562575941676792)\n",
      "LIT-SYN-All Dmetric: (0.7412060301507538, 0.7069408740359897, 0.7242693773824651)\n",
      "-------------- FOLD 10 ---------------\n",
      "Total time: 69.06761942803132, Average Time: 0.03336599972368663\n",
      "LIT-SYN-1 PCmetric: (0.9803921568627451, 0.9787234042553191, 0.9795918367346939)\n",
      "LIT-SYN-1 Dmetric: (0.86, 0.6304347826086957, 0.75)\n",
      "LIT-SYN-2 PCmetric: (0.9565217391304348, 1.0, 0.9773584905660377)\n",
      "LIT-SYN-2 Dmetric: (0.6893939393939394, 0.6220472440944882, 0.6563706563706564)\n",
      "LIT-SYN-3 PCmetric: (0.9119496855345912, 0.98125, 0.9467084639498433)\n",
      "LIT-SYN-3 Dmetric: (0.8068965517241379, 0.6560509554140127, 0.7284768211920529)\n",
      "LIT-SYN-8 PCmetric: (0.8214285714285714, 0.9066666666666666, 0.8616352201257862)\n",
      "LIT-SYN-8 Dmetric: (1.0144927536231885, 1.088235294117647, 1.051094890510949)\n",
      "LIT-SYN-All PCmetric: (0.9166666666666666, 0.9731051344743277, 0.9441141498216409)\n",
      "LIT-SYN-All Dmetric: (0.8106060606060606, 0.7160804020100503, 0.7632241813602015)\n",
      "------------ AVERAGE --------------\n",
      "Average, LIT-SYN-1, PCMetric - On: 98.8, Off: 99.6, Total: 99.2\n",
      "Average, LIT-SYN-1, DMetric - On: 0.8, Off: 0.6, Total: 0.7\n",
      "Average, LIT-SYN-2, PCMetric - On: 97.9, Off: 98.7, Total: 98.3\n",
      "Average, LIT-SYN-2, DMetric - On: 0.7, Off: 0.6, Total: 0.7\n",
      "Average, LIT-SYN-3, PCMetric - On: 94.2, Off: 98.3, Total: 96.3\n",
      "Average, LIT-SYN-3, DMetric - On: 0.8, Off: 0.8, Total: 0.8\n",
      "Average, LIT-SYN-8, PCMetric - On: 89.0, Off: 90.2, Total: 89.6\n",
      "Average, LIT-SYN-8, DMetric - On: 1.0, Off: 1.0, Total: 1.0\n",
      "Average, LIT-SYN-All, PCMetric - On: 94.8, Off: 96.9, Total: 95.8\n",
      "Average, LIT-SYN-All, DMetric - On: 0.8, Off: 0.8, Total: 0.8\n"
     ]
    }
   ],
   "source": [
    "from PostProcessing import PostProcessing\n",
    "from DataHandler import DataHandler\n",
    "\n",
    "postProcessing = PostProcessing(configs=configs)\n",
    "dataHandler = DataHandler(configs=configs)\n",
    "\n",
    "group_distribution = {\n",
    "    \"1\": 4139,\n",
    "    \"2\": 6916,\n",
    "    \"3\": 7128,\n",
    "    \"8\": 2629\n",
    "}\n",
    "\n",
    "general_qtd_train, general_qtd_test = dataHandler.generateAcquisitionType(trainSize, distribution=group_distribution)\n",
    "X_all = np.vstack((x_train, x_test))\n",
    "ydet_all = np.vstack((y_train[\"detection\"], y_test[\"detection\"]))\n",
    "ytype_all = np.vstack((y_train[\"type\"], y_test[\"type\"]))\n",
    "yclass_all = np.vstack((y_train[\"classification\"], y_test[\"classification\"]))\n",
    "\n",
    "general_qtd = np.vstack((np.expand_dims(general_qtd_train, axis=1), np.expand_dims(general_qtd_test, axis=1)))\n",
    "\n",
    "pcMetric, dMetric = [], []\n",
    "for fold in range(1, 11):\n",
    "    foldFolderPath = folderPath + str(fold) + \"/\"\n",
    "    \n",
    "    train_index = np.load(foldFolderPath + \"train_index.npy\")\n",
    "    test_index = np.load(foldFolderPath + \"test_index.npy\")\n",
    "\n",
    "    bestModel = ModelHandler.loadModel(foldFolderPath + \"best_model.h5\", type_weights=None) # Load model\n",
    "\n",
    "    x_train = X_all[train_index]\n",
    "    x_test = X_all[test_index]\n",
    "    y_train[\"detection\"] = ydet_all[train_index]\n",
    "    y_test[\"detection\"] = ydet_all[test_index]\n",
    "    y_train[\"type\"] = ytype_all[train_index]\n",
    "    y_test[\"type\"] = ytype_all[test_index]\n",
    "    y_train[\"classification\"] = yclass_all[train_index]\n",
    "    y_test[\"classification\"] = yclass_all[test_index]\n",
    "\n",
    "    general_qtd_test = general_qtd[test_index]\n",
    "\n",
    "    print(f\"-------------- FOLD {fold} ---------------\")\n",
    "    pcMetric_fold, dMetric_fold = postProcessing.checkModel(bestModel, x_test, y_test, general_qtd=general_qtd_test, print_error=False)\n",
    "    pcMetric.append(pcMetric_fold)\n",
    "    dMetric.append(dMetric_fold)\n",
    "\n",
    "print(\"------------ AVERAGE --------------\")\n",
    "avgPCMetric = np.average(pcMetric, axis=0) * 100\n",
    "avgDMetric = np.average(dMetric, axis=0)\n",
    "for i, subset in enumerate([\"1\", \"2\", \"3\", \"8\", \"All\"]):\n",
    "    print(f\"Average, LIT-SYN-{subset}, PCMetric - On: {avgPCMetric[i][0]:.1f}, Off: {avgPCMetric[i][1]:.1f}, Total: {avgPCMetric[i][2]:.1f}\")\n",
    "    print(f\"Average, LIT-SYN-{subset}, DMetric - On: {avgDMetric[i][0]:.1f}, Off: {avgDMetric[i][1]:.1f}, Total: {avgDMetric[i][2]:.1f}\")"
   ]
  }
 ]
}